using System;
using System.Threading.Tasks;
using System.Collections.Generic;
using System.Windows.Forms;
using MongoDB.Bson;
using MongoDB.Driver;
using System.Diagnostics;
using DnsClient.Protocol;
using System.Linq;
using System.Collections;
using System.Text.RegularExpressions;
using System.IO;
using System.Runtime.Serialization.Formatters.Binary;
using System.Threading;

namespace ScrapeSefariaMongoDBinDotNet
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
        }

        static string db_name = "safria";

        class book_info
        {
            public string name_eng;
            public string name_heb;
            public int[] chaps;
            public book_info(string _name_eng, string _name_heb, int[] _chaps)
            {
                name_eng = _name_eng;
                name_heb = _name_heb;
                chaps = _chaps;
            }
        }

        book_info[] tnc_db_array = new[] {
            new book_info("Genesis", "בראשית", new int[] { 31, 25, 24, 26, 32, 22, 24, 22, 29, 32, 32, 20, 18, 24, 21, 16, 27, 33, 38, 18, 34, 24, 20, 67, 34, 35, 46, 22, 35, 43, 54, 33, 20, 31, 29, 43, 36, 30, 23, 23, 57, 38, 34, 34, 28, 34, 31, 22, 33, 26 }),
            new book_info("Exodus", "שמות", new int[] { 22, 25, 22, 31, 23, 30, 29, 28, 35, 29, 10, 51, 22, 31, 27, 36, 16, 27, 25, 23, 37, 30, 33, 18, 40, 37, 21, 43, 46, 38, 18, 35, 23, 35, 35, 38, 29, 31, 43, 38 }),
            new book_info("Leviticus", "ויקרא", new int[] { 17, 16, 17, 35, 26, 23, 38, 36, 24, 20, 47, 8, 59, 57, 33, 34, 16, 30, 37, 27, 24, 33, 44, 23, 55, 46, 34 }),
            new book_info("Numbers", "במדבר", new int[] { 54, 34, 51, 49, 31, 27, 89, 26, 23, 36, 35, 16, 33, 45, 41, 35, 28, 32, 22, 29, 35, 41, 30, 25, 18, 65, 23, 31, 39, 17, 54, 42, 56, 29, 34, 13 }),
            new book_info("Deuteronomy", "דברים", new int[] { 46, 37, 29, 49, 29, 25, 26, 20, 29, 22, 32, 31, 19, 29, 23, 22, 20, 22, 21, 20, 23, 29, 26, 22, 19, 19, 26, 69, 28, 20, 30, 52, 29, 12 }),
            new book_info("Joshua", "יהושע", new int[] { 18, 24, 17, 24, 15, 27, 26, 35, 27, 43, 23, 24, 33, 15, 63, 10, 18, 28, 51, 9, 43, 34, 16, 33 }),
            new book_info("Judges", "שופטים", new int[] { 36, 23, 31, 24, 31, 40, 25, 35, 57, 18, 40, 15, 25, 20, 20, 31, 13, 31, 30, 48, 25 }),
            new book_info("I Samuel", "שמואל א", new int[] { 28, 36, 21, 22, 12, 21, 17, 22, 27, 27, 15, 25, 23, 52, 35, 23, 58, 30, 24, 42, 16, 23, 29, 22, 44, 25, 12, 25, 11, 31, 13 }),
            new book_info("II Samuel", "שמואל ב", new int[] { 27, 32, 39, 12, 25, 23, 29, 18, 13, 19, 27, 31, 39, 33, 37, 23, 29, 32, 44, 26, 22, 51, 39, 25 }),
            new book_info("I Kings", "מלכים א", new int[] { 53, 46, 28, 20, 32, 38, 51, 66, 28, 29, 43, 33, 34, 31, 34, 34, 24, 46, 21, 43, 29, 54 }),
            new book_info("II Kings", "מלכים ב", new int[] { 18, 25, 27, 44, 27, 33, 20, 29, 37, 36, 20, 22, 25, 29, 38, 20, 41, 37, 37, 21, 26, 20, 37, 20, 30 }),
            new book_info("Isaiah", "ישעיהו", new int[] { 31, 22, 26, 6, 30, 13, 25, 23, 20, 34, 16, 6, 22, 32, 9, 14, 14, 7, 25, 6, 17, 25, 18, 23, 12, 21, 13, 29, 24, 33, 9, 20, 24, 17, 10, 22, 38, 22, 8, 31, 29, 25, 28, 28, 25, 13, 15, 22, 26, 11, 23, 15, 12, 17, 13, 12, 21, 14, 21, 22, 11, 12, 19, 11, 25, 24 }),
            new book_info("Jeremiah", "ירמיהו", new int[] { 19, 37, 25, 31, 31, 30, 34, 23, 25, 25, 23, 17, 27, 22, 21, 21, 27, 23, 15, 18, 14, 30, 40, 10, 38, 24, 22, 17, 32, 25, 39, 44, 26, 22, 19, 32, 21, 28, 18, 16, 18, 22, 13, 30, 5, 28, 7, 47, 39, 46, 64, 34 }),
            new book_info("Ezekiel", "יחזקאל", new int[] { 28, 10, 27, 17, 17, 14, 27, 18, 11, 22, 25, 28, 23, 23, 8, 63, 24, 32, 14, 44, 37, 31, 49, 27, 17, 21, 36, 26, 21, 26, 18, 32, 33, 31, 15, 38, 28, 23, 29, 49, 26, 20, 27, 31, 25, 24, 23, 35 }),
            new book_info("Hosea", "הושע", new int[] { 9, 25, 5, 19, 15, 11, 16, 14, 17, 15, 11, 15, 15, 10 }),
            new book_info("Joel", "יואל", new int[] { 20, 27, 5, 21 }),
            new book_info("Amos", "עמוס", new int[] { 15, 16, 15, 13, 27, 14, 17, 14, 15 }),
            new book_info("Obadiah", "עובדיה", new int[] { 21 }),
            new book_info("Jonah", "יונה", new int[] { 16, 11, 10, 11 }),
            new book_info("Micah", "מיכה", new int[] { 16, 13, 12, 14, 14, 16, 20 }),
            new book_info("Nahum", "נחום", new int[] { 14, 14, 19 }),
            new book_info("Habakkuk", "חבקוק", new int[] { 17, 20, 19 }),
            new book_info("Zephaniah", "צפניה", new int[] { 18, 15, 20 }),
            new book_info("Haggai", "חגי", new int[] { 15, 23 }),
            new book_info("Zechariah", "זכריה", new int[] { 17, 17, 10, 14, 11, 15, 14, 23, 17, 12, 17, 14, 9, 21 }),
            new book_info("Malachi", "מלאכי", new int[] { 14, 17, 24 }),
            new book_info("Psalms", "תהילים", new int[] { 6, 12, 9, 9, 13, 11, 18, 10, 21, 18, 7, 9, 6, 7, 5, 11, 15, 51, 15, 10, 14, 32, 6, 10, 22, 12, 14, 9, 11, 13, 25, 11, 22, 23, 28, 13, 40, 23, 14, 18, 14, 12, 5, 27, 18, 12, 10, 15, 21, 23, 21, 11, 7, 9, 24, 14, 12, 12, 18, 14, 9, 13, 12, 11, 14, 20, 8, 36, 37, 6, 24, 20, 28, 23, 11, 13, 21, 72, 13, 20, 17, 8, 19, 13, 14, 17, 7, 19, 53, 17, 16, 16, 5, 23, 11, 13, 12, 9, 9, 5, 8, 29, 22, 35, 45, 48, 43, 14, 31, 7, 10, 10, 9, 8, 18, 19, 2, 29, 176, 7, 8, 9, 4, 8, 5, 6, 5, 6, 8, 8, 3, 18, 3, 3, 21, 26, 9, 8, 24, 14, 10, 8, 12, 15, 21, 10, 20, 14, 9, 6 }),
            new book_info("Proverbs", "משלי", new int[] { 33, 22, 35, 27, 23, 35, 27, 36, 18, 32, 31, 28, 25, 35, 33, 33, 28, 24, 29, 30, 31, 29, 35, 34, 28, 28, 27, 28, 27, 33, 31 }),
            new book_info("Job", "איוב", new int[] { 22, 13, 26, 21, 27, 30, 21, 22, 35, 22, 20, 25, 28, 22, 35, 22, 16, 21, 29, 29, 34, 30, 17, 25, 6, 14, 23, 28, 25, 31, 40, 22, 33, 37, 16, 33, 24, 41, 30, 32, 26, 17 }),
            new book_info("Daniel", "דניאל", new int[] { 21, 49, 33, 34, 30, 29, 28, 27, 27, 21, 45, 13 }),
            new book_info("Ezra", "עזרא", new int[] { 11, 70, 13, 24, 17, 22, 28, 36, 15, 44 }),
            new book_info("Nehemiah", "נחמיה", new int[] { 11, 20, 38, 17, 19, 19, 72, 18, 37, 40, 36, 47, 31 }),
            new book_info("Song of Songs", "שיר השירים", new int[] { 17, 17, 11, 16, 16, 12, 14, 14 }),
            new book_info("Ruth", "רות", new int[] { 22, 23, 18, 22 }),
            new book_info("Lamentations", "איכה", new int[] { 22, 22, 66, 22, 22 }),
            new book_info("Ecclesiastes", "קהלת", new int[] { 18, 26, 22, 17, 19, 12, 29, 17, 18, 20, 10, 14 }),
            new book_info("Esther", "אסתר", new int[] { 22, 23, 15, 17, 14, 14, 10, 17, 32, 3 }),
            new book_info("I Chronicles", "דברי הימים א", new int[] { 54, 55, 24, 43, 41, 66, 40, 40, 44, 14, 47, 41, 14, 17, 29, 43, 27, 17, 19, 8, 30, 19, 32, 31, 31, 32, 34, 21, 30 }),
            new book_info("II Chronicles", "דברי הימים ב", new int[] { 18, 17, 17, 22, 14, 42, 22, 18, 31, 19, 23, 16, 23, 14, 19, 14, 19, 34, 11, 37, 20, 12, 21, 27, 28, 23, 9, 27, 36, 27, 21, 33, 25, 33, 27, 23 })
        };

        static Hashtable torah_books_ht = new Hashtable();
        static Hashtable nach_books_ht = new Hashtable();

        static string[] my_selected_trans_books_ar = { "Onkelos ", "Targum Jonathan on ", "Targum of " /* Targum Jonathan only for I,II Chronicles */, "Targum Jerusalem, ", "Rashi on ", "Ibn Ezra on ", "Ramban on ", "Rashbam on ", "Chizkuni, ", "Daat Zkenim on ", "Sforno on ", "Gur Aryeh on ", "Or HaChaim on ", "Kli Yakar on ", "Haamek Davar on ", "Radak on ", "Ralbag ", "Metzudat David on ", "Metzudat Zion on ", "Malbim on ", "Malbim Beur Hamilot on " };
        class t_nach_counts
        {
            public int t = 0;
            public int nach = 0;
        }
        static Hashtable t_nach_counts_ht = new Hashtable();

        private void Form1_Load(object sender, EventArgs e)
        {
            Task.Run(() => connect_to_mongodb_client_async()).Wait(); // This is not working: connect_to_mongodb_client_async().Wait();
            int i;
            for (i = 0; i < 5; i++)
            {
                torah_books_ht.Add(tnc_db_array[i].name_eng, null);
            }
            for (i = 5; i < tnc_db_array.Length; i++)
            {
                nach_books_ht.Add(tnc_db_array[i].name_eng, null);
            }
            for (i = 0; i < my_selected_trans_books_ar.Length; i++)
            {
                t_nach_counts_ht.Add(my_selected_trans_books_ar[i], new t_nach_counts());
            }
        }

        static IMongoDatabase db;
        static IMongoCollection<BsonDocument> links_collection;
        static IMongoCollection<BsonDocument> texts_collection;
        static IMongoCollection<BsonDocument> index_collection;

        static async Task connect_to_mongodb_client_async()
        {
            MongoClient dbClient = new MongoClient("mongodb://localhost:27017");
            db = dbClient.GetDatabase(db_name);
            links_collection = db.GetCollection<BsonDocument>("links");
            texts_collection = db.GetCollection<BsonDocument>("texts");
            index_collection = db.GetCollection<BsonDocument>("index");
        }

        private void button2_Click(object sender, EventArgs e)
        {
            get_verse_with_nikkud("Genesis", 0, 0);
        }

        static string get_verse_with_nikkud(string book_name, int chap_n, int verse_n)
        {
            FilterDefinition<BsonDocument> filter = "{ title: '" + book_name + "', language: 'he', versionTitle: 'Tanach with Nikkud' }";
            string verse = (string)texts_collection.Find(filter).FirstOrDefault()["chapter"][chap_n][verse_n];
            return verse;

            //IAsyncCursor<BsonDocument> cursor = books_collection.FindSync(filter);
            //await cursor.MoveNextAsync();
            //int n_boooks_results = cursor.Current.Count();
            //if (n_boooks_results != 1) { throw new Exception($"Expecting 1 book result, but got: {n_boooks_results} results."); }
            //string verse2 = (string)cursor.Current.First()["chapter"][chap_n][verse_n];
        }

        private void button3_Click(object sender, EventArgs e)
        {
            bool _load = false;
            if (_load)
            {
                FileStream _load_fs = new FileStream("torah_ref_books_ht.ser", FileMode.Open, FileAccess.Read);
                torah_ref_books_ht = (Hashtable)(new BinaryFormatter()).Deserialize(_load_fs);
                _load_fs = new FileStream("torah_ref_books_ht.ser", FileMode.Open, FileAccess.Read);
                bible_ref_books_ht = (Hashtable)(new BinaryFormatter()).Deserialize(_load_fs);
            }

            bool log_details = true;
            Console.WriteLine("Scanning All Bible Verses Translation (only references without content) from Safria has Started.");
            //Thread.Sleep(2000);
            Stopwatch bible_scanning_timer = new Stopwatch();
            bible_scanning_timer.Start();
            int bible_n_verses = 0;
            int bible_n_refs = 0;
            for (int book_i = 0; book_i < tnc_db_array.Length; book_i++)
            {
                book_info book = tnc_db_array[book_i];
                Stopwatch book_scanning_timer = new Stopwatch();
                book_scanning_timer.Start();
                string book_name = book.name_eng;
                int book_n_verses = 0;
                int book_n_refs = 0;
                for (int ch_n = 1; ch_n <= book.chaps.Length; ch_n++)
                {
                    for (int v_n = 1; v_n <= book.chaps[ch_n - 1]; v_n++)
                    {
                        book_n_verses++;
                        bible_n_verses++;

                        Stopwatch verse_timer = new Stopwatch();
                        verse_timer.Restart();

                        int n_refs = get_verse_refs(book_name, ch_n, v_n, log_details);

                        book_n_refs += n_refs;
                        bible_n_refs += n_refs;

                        bool _report = false;
                        if (_report)
                        {
                            foreach (var ref_book_name in nach_ref_books_ht.Keys)
                            {
                                Console.WriteLine(ref_book_name + "\t" + nach_ref_books_ht[ref_book_name]);
                            }
                        }
                        // Console.WriteLine($"Scanning Verse {book_name}:{ch_n}:{v_n} found {n_refs} references at: {verse_timer.Elapsed.ToString(@"mm\:ss")}");
                        bool _save = false;
                        if (_save)
                        {
                            FileStream file_write_stream = System.IO.File.Open("bible_ref_books_ht.ser", FileMode.Create, FileAccess.Write);
                            (new BinaryFormatter()).Serialize(file_write_stream, bible_ref_books_ht);
                            file_write_stream.Close();
                        }
                    }
                }
                //book_scanning_timer.Stop();
                Console.WriteLine($"Scanning the book: {book_name} with {book_n_verses} verses, got: {book_n_refs} references, took: {book_scanning_timer.Elapsed.ToString(@"mm\:ss")} sec.");
            }
            Console.WriteLine($"Scanning the whole Bible with {bible_n_verses} verses, got: {bible_n_refs} references, took: {bible_scanning_timer.Elapsed.ToString(@"mm\:ss")} sec.");
            //
            foreach (string my_selected_trans_book in my_selected_trans_books_ar)
            {
                t_nach_counts _t_nach_counts = ((t_nach_counts)t_nach_counts_ht[my_selected_trans_book]);
                Console.WriteLine($"{my_selected_trans_book}{_t_nach_counts.t} {_t_nach_counts.nach}");
            }
        }

        static Hashtable bible_ref_books_ht = new Hashtable();
        static Hashtable torah_ref_books_ht = new Hashtable();
        static Hashtable nach_ref_books_ht = new Hashtable();
        static Hashtable total_ref_books_missed_ht = new Hashtable();
        static Hashtable ref_bible_ht = new Hashtable();

        static int get_verse_refs(string book_name, int chap_n, int verse_n, bool log_details)
        {
            // FilterDefinition<BsonDocument> refs_filter = "{ $expr: { $let: { vars: { fst: { $arrayElemAt: [ \"$refs\", 0 ] } }, in: { $eq: [ \"$$fst\", \"" + book_name  + " " + chap_n + ":" + verse_n + "\" ] } } } }";
            string verse_str = $"{book_name} {chap_n}:{verse_n}";

            FilterDefinition<BsonDocument> refs_filter;
            if (true)
            {
                refs_filter = "{ $or: [{ expandedRefs0: '" + verse_str + "'}] }"; // , { expandedRefs1: '"+ verse_str + "'}

            }
            else
            {
                // string _trans = "Rashi" + " on ";
                string _trans = "Chizkuni," + " ";
                string _str = "{ $and: [ {expandedRefs0: \"" + verse_str + "\"}, { refs: { $regex : \"^" + _trans + verse_str + ":\" } } ] }";
                refs_filter = _str;
            }


            // FilterDefinition<BsonDocument> refs_filter = $"{{ $or: [{{expandedRefs0: \"\"}}, {{ expandedRefs1: \"{book_name} {chap_n}:{verse_n}\"}} }} ] }}";
            IAsyncCursor<BsonDocument> refs_cursor = links_collection.FindSync(refs_filter);
            //
            //Console.WriteLine($"Links for: {book_name}:{chap_n }:{verse_n}");
            int n_refs = 0;
            int n_err = 0;
            Hashtable ref_books_ht = new Hashtable();
            int n_ref_books = 0;
            Stopwatch doc_timer = new Stopwatch();
            doc_timer.Restart();
            Hashtable results = new Hashtable();
            //string[] my_selected_trans_books_ar = { "" };
            while (refs_cursor.MoveNext())
            {
                IEnumerable<BsonDocument> batch = refs_cursor.Current;

                foreach (BsonDocument document in batch)
                {
                    if (!(document["availableLangs"].ToString()).Contains("he"))
                    {
                        // Warning
                    }
                    BsonArray ref_ar = (BsonArray)document["refs"];
                    string _ref = (string)ref_ar[1];

                    bool found = false;
                    Match _m = null;
                    foreach (string my_trans in my_selected_trans_books_ar)
                    {
                        if (_ref.StartsWith("Gur Aryeh"))
                        {
                            // debug.
                        }
                        _m = Regex.Match(_ref, $"^(?<t_book>{my_trans}{book_re} {chap_n}:{verse_n})(:(?<t_i>[\\d]+))?$");
                        if (_m.Success)
                        {
                            found = true;
                            break;
                        }
                    }
                    if (!found)
                    {
                        continue;
                    }

                    n_refs++;

                    if (!results.Contains(_ref))
                    {
                        string sub_section_str = _m.Groups["t_i"].Value;
                        int sub_section_i = 0; // has no sub-sections.
                        if (sub_section_str != "")
                        {
                            sub_section_i = int.Parse(sub_section_str);
                        }
                        var key = _m.Groups["t_book"].Value;
                        if (!results.ContainsKey(key))
                        {
                            results.Add(key, new SortedList<int, string>());
                        }
                        SortedList<int, string> sl = (SortedList<int, string>)results[key];
                        if (sl.ContainsKey(sub_section_i))
                        {
                            n_err++;
                            Console.WriteLine("Err: reference reapears: " + _ref);
                        }
                        else
                        {
                            sl.Add(sub_section_i, _ref);
                        }
                    }
                    else
                    {
                        // error?
                        var _old_doc = results[_ref];
                    }
                }
            }

            foreach (string my_selected_trans_book in my_selected_trans_books_ar)
            {
                var res_key_refs = (from DictionaryEntry de in results where ((string)de.Key).StartsWith(my_selected_trans_book) select de);
                if (res_key_refs.Count() == 0)
                {
                    // no translation with this book.
                    continue;
                }
                if (res_key_refs.Count() != 1)
                {
                    throw new Exception();
                }
                n_ref_books += 1;
                string res_key_ref = (string)res_key_refs.First().Key;

                int ref_n_sections = ((SortedList<int, string>)results[res_key_ref]).Count;
                t_nach_counts _t_nach_counts = ((t_nach_counts)t_nach_counts_ht[my_selected_trans_book]);
                if (torah_books_ht.ContainsKey(book_name))
                {
                    _t_nach_counts.t++;
                }
                else if (nach_books_ht.ContainsKey(book_name))
                {
                    _t_nach_counts.nach++;
                }
                else
                {
                    throw new Exception();
                }

                bool short_description_log = true;
                if (short_description_log)
                {
                    Console.WriteLine($"({ref_n_sections}) {res_key_ref}");
                }
                else
                {
                    foreach (KeyValuePair<int, string> kvp in (SortedList<int, string>)results[res_key_ref])
                    {
                        string _ref = kvp.Value;
                        string content = get_content(book_name, chap_n, verse_n, verse_str, ref n_err, ref_books_ht, _ref);
                        if (content == null)
                        {
                            continue;
                            throw new Exception("Couldn't get Content of " + (string)res_key_ref);
                        }
                        content = content.Trim();
                        if (log_details)
                        {
                            Console.WriteLine($"({kvp.Key}){_ref} : {content}"); // ({doc_timer.Elapsed.ToString(@"mm\:ss")}):
                        }

                    }
                }
            }

            if (true || log_details)
            {
                string avg = ((float)doc_timer.Elapsed.Milliseconds / n_refs / 1000).ToString("0.0000");
                string err_str = n_err == 0 ? "" : $" n_err: {n_err}";
                if (false && n_ref_books != ref_books_ht.Keys.Count) throw new Exception();
                Console.WriteLine($"found {n_refs} References in {n_ref_books} Books for {book_name}:{chap_n }:{verse_n}{err_str}"); //  time: {doc_timer.Elapsed.ToString(@"mm\:ss")} , avg. {avg} sec.
            }
            return n_refs;
        }
    }
}
